package com.chance.cc.crawler.development.bootstrap.pdd;

import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * @author bx
 * @date 2021/4/27 0027 12:51
 */
public class PDDTest2 {

    public static Logger logger = LoggerFactory.getLogger(PDDTest2.class);

    public static void main(String[] args) throws IOException, InterruptedException {
        compare();

    }

    private static void washMall() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
        BufferedReader bufferedReader = new BufferedReader(new FileReader("D:\\chance\\data\\pdd\\all\\pdd_product_all.json"));
        FileOutputStream mallOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_mallId.json");
        String line;
        Set<String> mallIds = new HashSet<>();
        while ((line = bufferedReader.readLine())!= null){
            Json lineJson = new Json(line);
            try {
                String mall_id = lineJson.jsonPath("$.mall_id").get();
                mallIds.add(mall_id);
            } catch (Exception e) {
                try {
                    String mall_id = lineJson.jsonPath("$.item_data.goods_model.mall_id").get();
                    mallIds.add(mall_id);
                } catch (Exception ex) {
//                    System.out.println("error line :" + line);
                }

            }
        }
        IOUtils.writeLines(mallIds,"\r\n",mallOutputStream,"utf-8");
        System.out.println(mallIds.size());

    }

    private static void compare() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);

        List<String> mallIds = IOUtils.readLines(new FileInputStream("D:\\chance\\data\\pdd\\pdd_mallId.json"), "UTF-8");
        Set<String> smembers = redisReader.smembers("pdd-mall-id-queue_set");
        List<String> results = new ArrayList<>();
        for (String mallId : mallIds) {
            if (!smembers.contains( mallId)){
                redisReader.sadd("pdd-mall-id-queue_set",mallId);
                redisReader.listPush("pdd-mall-id-queue_liat",mallId);
            }
        }
        System.out.println(results.size());
    }

    private static void statics() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
        FileOutputStream productOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_product_id.json", true);

        Set<String> fenlei = redisReader.smembers("pdd-fenlei-goods-id-queue_set");
        Set<String> html = redisReader.smembers("pdd-html-shop-goods-id-queue_set");
        Set<String> goods = redisReader.smembers("pdd-goods-id-queue_set");
        Set<String> results = new HashSet<>();
        results.addAll(fenlei);
        results.addAll(html);
        results.addAll(goods);
        System.out.println(results.size());
        IOUtils.writeLines(results,"\r\n",productOutputStream,"utf-8");
    }

    private static void merge() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
        FileOutputStream productOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\all\\pdd_product_all.json", true);
        Set<String> productIds = new HashSet<>();
        List<String> results = new ArrayList<>();

        merge(results,"D:\\chance\\data\\pdd\\pdd.json",productIds,productOutputStream);
        merge(results,"D:\\chance\\data\\pdd\\pdd_html_shop.json",productIds,productOutputStream);
        merge(results,"D:\\chance\\data\\pdd\\pdd_subfenlei.json",productIds,productOutputStream);

        if (results.size() >0){
            IOUtils.writeLines(results,"\r\n",productOutputStream,"utf-8");
        }
        System.out.println(productIds.size());
    }

    private static void merge(List<String> results,String FilePath,
                              Set<String> productIds,FileOutputStream productOutputStream) throws IOException {
        BufferedReader bufferedReader = new BufferedReader(new FileReader(FilePath));

        String line;
        while ((line = bufferedReader.readLine())!= null){
            try {
                Json lineJson = new Json(line);
                String productId;
                if (!line.contains("\"item_data\":")){
                    productId = lineJson.jsonPath("$.goods_id").get();
                } else {
                    productId = lineJson.jsonPath("$.item_data.goods_model.goods_id").get();
                }

                if (!productIds.contains(productId)){
                    productIds.add(productId);
                    results.add(line);
                }

                if (results.size() > 1000){
                    IOUtils.writeLines(results,"\r\n",productOutputStream,"utf-8");
                    results.clear();
                }
            } catch (Exception e) {
                System.out.println("error line :" + line);
            }
        }

    }
    private static void statics2() throws IOException {
        BufferedReader bufferedReader = new BufferedReader(new FileReader("D:\\chance\\data\\pdd\\all\\pdd_product_all.json"));
        Set<String> productIds = new HashSet<>();
        String line;
        while ((line = bufferedReader.readLine())!= null){
            Json lineJson = new Json(line);
            try {
                String productId;
                if (!line.contains("\"item_data\":")){
                    productId = lineJson.jsonPath("$.goods_id").get();
                } else {
                    productId = lineJson.jsonPath("$.item_data.goods_model.goods_id").get();
                }

                if (!productIds.contains(productId)){
                    productIds.add(productId);
                } else {
                    System.out.println(productId);
                }

            } catch (Exception e) {
                System.out.println("error line :" + line);
            }
        }
        System.out.println(productIds.size());
    }
}
